#!/usr/bin/env python
# -*- coding: utf-8 -*-

import curl_async
import re
import json
import datetime
from tornado import httpclient, gen, ioloop, queues
# from tornadouvloop import TornadoUvloop
from store_info import crawl_task
from store_info import user
from store_info import comment
from store_info import relationship

crawl_task_db = crawl_task()
user_db = user()
comment_db = comment()
relationship_db = relationship()


async def get_task():
    """
    发送分页任务
    """
    task_query_info = {
        'type': 'topic',
        'status': 0,
    }
    base_url = ''
    status = False
    task_info = await crawl_task_db.pop_one_task(task_query_info)
    # 获取不到任务则休眠10秒
    if not task_info:
        print('empty task_info will sleep 10 second')
        await gen.sleep(10)
        return

    print('start task %s' % task_info['url'] if 'url' in task_info else '')
    if task_info and 'url' in task_info:
        base_url = task_info['url']
        request_status = await get_page(base_url)
        if request_status:
            task_info['status'] = 2
        else:
            task_info['status'] = -1
    else:
        task_info['status'] = -1
    status = task_info['status']
    # print('task_info end')
    # print(task_info)
    status = await crawl_task_db.save(task_info)
    print('end task %s :%s:' % (task_info['url'] if 'url' in task_info else '', status))


async def get_page(url):
    """
    获取对应url的数据
    """
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, sdch',
        'Accept-Language': 'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,zh-TW;q=0.2',
        'Connection': 'keep-alive',
        'Cookie': '',
        'Host': 'tieba.baidu.com',
        'Referer': 'http://tieba.baidu.com/mo/q---47BA1EC629DBB22F16C029A68019A549%3AFG%3D1--1-3-0--2--wapp_1439461323961_399/m?kw=%E6%B5%B7%E5%8D%97%E5%A4%A7%E5%AD%A6&lp=5011&lm=&pn=10',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36',
    }

    response = await curl_async.fetch(url, headers=headers)
    data = {}
    if response and response.code == 200:
        data = await re_page_info(response.body, url)
    else:
        return False
    if not data:
        return False
    re_tid = re.compile('p/(\d+)')
    tid = re_tid.findall(url)
    tid = tid[0]

    comment_list = await merge_comment_data(data, tid)
    user_list = await merge_user_data(data)
    await save_user_info(user_list)
    await save_comment_info(comment_list)
    return True



async def re_page_info(html, url):
    if not html:
        return False
    str_html = html
    html = html.decode('UTF-8', 'ignore')
    # 获取分页数量
    re_pn_total = re.compile('<a href="([^\?]*)\?pn=([^"]*)">尾页</a>')
    pn_total_info = re_pn_total.findall(html)

    # 发送分页任务
    if (pn_total_info):
        pn_url = ''
        total = 0
        pn_url, total = pn_total_info[0]
        total = int(total) + 1
        re_now_pn = re.compile('<span class="tP">(\d+)')
        now_pn_info = re_now_pn.findall(html)
        # 只有当前页为第一页，且存在多个分页，才发送分页任务
        if (total > 0) and (now_pn_info) and (now_pn_info[0] == '1'):
            for i in range(2, total):
                task_info = {
                    'type': 'topic',
                    'url': 'http://tieba.baidu.com'+pn_url + '?pn='+str(i),
                    'title': 'http://tieba.baidu.com'+pn_url + '?pn='+str(i),
                }
                await send_pn_task(task_info)
    # 回复用户
    re_user = re.compile('&ie=utf-8&fr=pb&ie=utf-8" target="_blank">([^<]*)</a>')
    user_list = re_user.findall(html)
    # 回复用户id
    re_user_ids = re.compile('<li class="d_name" data-field=\'{&quot;user_id&quot;:([^}]*)}\'>')
    user_ids = re_user_ids.findall(html)
    # 回复时间
    re_comment_date = re.compile('date&quot;:&quot;([^&]*)&quot;')
    comment_date_list = re_comment_date.findall(html)
    # 回复内容
    re_comment = re.compile('<div id="post_content_([^"]*)" class="([^"]*)">            ([^<]*)')
    comment_list = re_comment.findall(html)
    # 本页所有用户列表提取
    re_sub_user_list = re.compile('"userList" : (.*?),    "commentList" :')
    sub_user_list = re_sub_user_list.findall(html)
    if sub_user_list:
        sub_user_list = json.loads(sub_user_list[0])
    else:
        sub_user_list = []

    # 楼中楼内容第一页提取
    re_sub_comment_list = re.compile('"commentList" : (.*?)\]},')
    sub_comment_list = re_sub_comment_list.findall(html)
    if sub_comment_list:
        sub_comment_list[0] = sub_comment_list[0] + ']}'
        sub_comment_list = json.loads(sub_comment_list[0])
    else:
        sub_comment_list = []

    data = {
        'user_ids': user_ids,
        'user_list': user_list,
        'comment_date_list': comment_date_list,
        'comment_list': comment_list,
        'sub_user_list': sub_user_list,
        'sub_comment_list': sub_comment_list,
    }
    return data

async def merge_user_data(data):
    """
    合并用户列表
    merge_comment_data = {
        "uid1":{
            "uid" : 用户id,
            "username" : username,
            "thumb":用户头像标识,
        },
        "uid2":{
            "uid" : 用户id,
            "username" : username,
            "thumb":用户头像标识,
        }
    }
    """
    merge_user_list = {}
    length = len(data['user_list'])
    user_ids_length = len(data['user_ids'])
    for i in range(length):
        if (i <= user_ids_length) and data['user_ids'][i] != 'null':
            merge_user_list[data['user_ids'][i]] = {
                'uid': data['user_ids'][i],
                'username': data['user_list'][i]
            }
        else:
            merge_user_list[data['user_list'][i]] = {
                'uid': '',
                'username': data['user_list'][i]
            }

    for uid in data['sub_user_list']:
        merge_user_list[uid] = {
            "uid": uid,
            "username": data['sub_user_list'][uid]['user_name'],
            "thumb": 'http://tb.himg.baidu.com/sys/portrait/item/' + data['sub_user_list'][uid]['portrait'],
        }
    return merge_user_list

async def merge_comment_data(data, tid):
    """
    合并返回数据
    merge_comment_data = [{
        "tid":主题id,
        "pid":父级回复id,
        "comment_id":回复id,
        "content":回复内容,
        "username":username,
        "uid":用户id,
        "date":回复时间,
        },{
        "tid":主题id,
        "pid":父级回复id,
        "comment_id":回复id,
        "content":回复内容,
        "username":username,
        "uid":用户id,
        "date":回复时间,
        }]
    """
    merge_comment_data = []
    comment_report = {}

    length = len(data['comment_date_list'])
    user_ids_length = len(data['user_ids'])
    # 合并主楼评论
    for i in range(length):
        comment_info = {
            "tid": tid,
            "pid": 0,
            "comment_id": int(data['comment_list'][i][0]),
            "content": data['comment_list'][i][2],
            "username": data['user_list'][i],
            "uid": data['user_ids'][i] if (i <= user_ids_length and data['user_ids'][i] != 'null') else '',
            "date": data['comment_date_list'][i],
        }
        # user comment sum
        if comment_info['username'] in comment_report:
            comment_report[comment_info['username']] = comment_report[comment_info['username']] + 1
        else:
            comment_report[comment_info['username']] = 1
        merge_comment_data.append(comment_info)
    # 合并楼中楼评论
    for parent_comment_id in data['sub_comment_list']:
        for sub_comment_info in data['sub_comment_list'][parent_comment_id]:
            comment_date = datetime.datetime.utcfromtimestamp(sub_comment_info['now_time'])
            comment_time = "%04d-%02d-%02d %02d:%02d:%02d" % (comment_date.year, comment_date.month, comment_date.day, comment_date.hour, comment_date.minute, comment_date.second)
            comment_info = {
                "tid": tid,
                "pid": parent_comment_id,
                "comment_id": int(sub_comment_info['comment_id']),
                "content": sub_comment_info['content'],
                "username": data['sub_user_list'][sub_comment_info['user_id']]['user_name'],
                "uid": sub_comment_info['user_id'],
                "date": comment_time,
            }
            # user comment sum
            if comment_info['username'] in comment_report:
                comment_report[comment_info['username']] = comment_report[comment_info['username']] + 1
            else:
                comment_report[comment_info['username']] = 1
            merge_comment_data.append(comment_info)
    await save_relationship_info(comment_report)
    return merge_comment_data

async def save_relationship_info(comment_report):
    for from_user in comment_report:
        for to_user in comment_report:
            relationship_info = {
                'from': from_user,
                'to': to_user,
                'count': comment_report[from_user],
            }
            # print(relationship_info)
            status = await relationship_db.save(relationship_info)
            # print('save_relationship_status: %s' % status, relationship_info)

async def save_user_info(user_list):
    for uid in user_list:
        # print(user_list[uid])
        status = await user_db.save(user_list[uid])
        # print('save_user_status: %s' % status, user_list[uid])

async def save_comment_info(comment_list):
    for comment_info in comment_list:
        status = False
        status = await comment_db.save(comment_info)
        # print('save_comment_status: %s' % status, comment_info)

async def send_pn_task(task_info):
    """
    发送主题分页任务
    """
    if task_info:
        # print(task_info)
        status = await crawl_task_db.save(task_info)
        # print('save_task_status: %s' % status, task_info)
    return
async def send_next_task():
    """
    发送下一个任务的信息
    """
    pass


async def main():
    while 1:
        await get_task()

if __name__ == '__main__':
    # ioloop.IOLoop.configure(TornadoUvloop)
    io_loop = ioloop.IOLoop.current()
    # 使用lambda封装传递参数
    io_loop.run_sync(lambda: main())
